 package util;


import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.InetAddress;
import java.net.Socket;
import java.net.UnknownHostException;

import org.htmlparser.Parser;
import org.htmlparser.filters.TagNameFilter;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;

/*
 * Created on 06/04/2006
 * This program read html page specified in first parameter
  * Window - Preferences - Java - Code Style - Code Templates
 */

/**
 * @author Grinfeld Igor. Project.
 * Window - Preferences - Java - Code Style - Code Templates
 */

//This class read html page from source URL and write it to output file. 
public class HtmlReader {

//Attributes
	protected String inURL=null;
	protected String outFile=null;
	protected int timeOut=60000;
	public boolean charsetCheck=false;//if true, perform charset check before reading
    protected String srcEncoding=null;//Encoding of source file    
	
//Methods
	public static void main(String args[])
	{
		HtmlReader reader=new HtmlReader(args[0],args[1]);
		reader.read(null);
	}
		
    //Constructor. First argument is source url, second is name of result file
	public HtmlReader(String args0,String args1)
	{
		inURL=args0;
		outFile=args1;		
	}
	
	//get curPage URL without http and returns String with body of html file
	public static String readHtmlBody(String curPage,String srcEncoding)
	{
		
		//Read main page of selled apartments from internet
		HtmlReader reader=
			new HtmlReader(curPage,"out.html");
		reader.charsetCheck=false;
		reader.setCharset(srcEncoding);
		String sTemp=reader.read(null);
		if(sTemp==null)
			return null;
			
				    
	    //extract html body
		try{
			Parser parser=new Parser();
			parser.setInputHTML(sTemp);		
		
			TagNameFilter tFilter=new TagNameFilter("body");
			NodeList list=parser.extractAllNodesThatMatch(tFilter);
			sTemp=list.elementAt(0).toHtml().toUpperCase();
		}
		catch(ParserException e)
		{
			System.out.println("Cannot parse this file: http://"+curPage);
			e.printStackTrace();
			return null;
			//MainFrame.msg="Cannot parse this file: http://"+curPage;
		}
		
	    //remove scripts
		StringBuffer sb = new StringBuffer();
		String token;
		int pos=sTemp.indexOf("<SCRIPT"),pos2=0;
		while(pos>=0)
		{																
			token=sTemp.substring(pos2,pos);
			sb.append(token);
			pos2=sTemp.indexOf("</SCRIPT>",pos)+9;								
			pos=sTemp.indexOf("<SCRIPT",pos2);
		}
	    sTemp=sb.toString();
	    
	    //remove comments
	    int len=sTemp.length();
	    sb = new StringBuffer();												
		pos2=sTemp.indexOf("<!--");
		if(pos2<0)
			pos2=len-1;		
		pos=0;
		while(pos>=0 && pos2>=0 && pos<len)
		{	
			token=sTemp.substring(pos,pos2);				
			sb.append(token);
			
			pos=sTemp.indexOf("-->",pos2);
			if(pos>=0)
			{
				pos+=3;
				pos2=sTemp.indexOf("<!--",pos);
				if(pos2<0)
					pos2=len-1;
			}
		}
	    sTemp=sb.toString();
	    return sTemp;				
	}
	
	//get Methods for class attributes
	public String getOutFile(){return outFile;}
	public String getSrcURL(){return inURL;}
	public String getCharset()
	{
		return srcEncoding==null ? null : srcEncoding;
	}

    public void setCharset(String val)
	{
		if(val!=null)
			srcEncoding=val;
	}
	
	// This method reads html page from URL specified in 'inURL' attribute
	// if parameter is null, result will be written to file with name of outFile
	// attribute, else it will be written to outFile parameter and outFile 
	// attribute will be updated to this value.
	public String read(String outFile)
	{
		if(outFile==null)
			outFile=this.outFile;
		else
			this.outFile=outFile;
		
		StringBuffer sTextFromDataArea=new StringBuffer();
		String sServerToUserLine;
		String sHost = inURL.substring(0,inURL.indexOf("/"));
		
		String sPage=inURL.substring(inURL.indexOf("/"));
		Socket sSocket;
		PrintWriter pwOut;
		BufferedReader brIn;
		int HTTP_PORT=80;

		try
		{
			InetAddress iaMyHostAddress=
				InetAddress.getByName(sHost);
    
			if(charsetCheck)
			{
				sSocket=new Socket(iaMyHostAddress, HTTP_PORT);
				sSocket.setSoTimeout( timeOut);
				pwOut=new PrintWriter(sSocket.getOutputStream(),true);
				brIn=new BufferedReader(
						new InputStreamReader(sSocket.getInputStream()));            
  
				//Send 'Get' request for encoding check
				pwOut.println("GET "+sPage+" HTTP/1.1");
				pwOut.println("Host: "+sHost);
				pwOut.println("User-Agent: Explorer/5.0 ");
				pwOut.println();
            
				//checking for response
				sServerToUserLine=brIn.readLine();                
            
				if(sServerToUserLine==null || !sServerToUserLine.contains("200"))
				{
					System.err.println("Failed to connect to the Server for charset check : "+sHost+"/"+sPage );
					return null;
				}      
  
				//check encoding of page
				sSocket.setSoTimeout(timeOut/5);
				int i;
				do{
	            	sServerToUserLine=brIn.readLine();
					
					//if we can't read next line more then timeOut, then exit
					if(sServerToUserLine==null)
					{
						brIn.close();
						pwOut.close();
						System.err.println("Read error. Cannot continue reading : "+sHost+"/"+sPage);
						return null;
					}
                
					sServerToUserLine=sServerToUserLine.toUpperCase().trim();
					i=sServerToUserLine.indexOf("CHARSET=");
					if(sServerToUserLine.startsWith("<META") && i>=0)
					{
						srcEncoding= sServerToUserLine.substring(i+8);
						i=srcEncoding.indexOf(";");
						if(i<0)
							i=srcEncoding.indexOf("'");
						if(i<0)
							i=srcEncoding.indexOf("\"");					
						if(i<0)
							i=srcEncoding.indexOf(" ");
						srcEncoding=srcEncoding.substring(0,i);
					}																
				}while(srcEncoding==null && !sServerToUserLine.contains("</html>"));
				brIn.close();
				pwOut.close();
			}
  
        	//reopen connection with correct encoding charset
        	sSocket=new Socket(iaMyHostAddress, HTTP_PORT);
			sSocket.setSoTimeout(timeOut);
        	pwOut=new PrintWriter(sSocket.getOutputStream(),true);
        	if(srcEncoding!=null)
        		brIn=new BufferedReader(
                   new InputStreamReader(sSocket.getInputStream(),srcEncoding));
        	else
        		brIn=new BufferedReader(
                        new InputStreamReader(sSocket.getInputStream()));
  
            //Send 'Get' request
            pwOut.println("GET "+sPage+" HTTP/1.1");
            pwOut.println("Host: "+sHost);
            pwOut.println("User-Agent: Explorer/5.0 ");
            pwOut.println();
            
            //checking response
            sServerToUserLine=brIn.readLine();                
            
            if( sServerToUserLine==null || !sServerToUserLine.contains("200"))
            {
            	System.err.println("Failed to connect to the Server : "+sPage);
                return null;
            }      
  
            //page reading
            sSocket.setSoTimeout(timeOut/5);
            do{
               	sServerToUserLine=brIn.readLine();
                
                //if we can't read next line more then timeOut, then exit
                if(sServerToUserLine==null)
                {
                	brIn.close();
                	pwOut.close();
					System.err.println("Read Error. Cannot continue reading : "+sHost+sPage);
                	return null;
                }
                
                if(sServerToUserLine.equals("-->") || sServerToUserLine.length()>3)
                	sTextFromDataArea=
                		sTextFromDataArea.append(sServerToUserLine);
            }while(!sServerToUserLine.equalsIgnoreCase("</html>"));
                        
            //Printing read html page to file
            PrintWriter prOut=new PrintWriter(outFile);
            prOut.println(sTextFromDataArea);
            prOut.close();
            
            //Closing socket
            pwOut.close();
            brIn.close();            
            sSocket.close();         
        } // End Of Try
        catch(UnknownHostException e)
        {
        	//MainFrame.msg=new String("Don't know about host");
        	System.err.println("Don't know about host : "+sHost+sPage);
        	e.printStackTrace();
            return null;
        }
        catch (IOException e)
        {
        	System.err.println("Cannot get IO for the connection : "+sHost+sPage);
        	e.printStackTrace();
            return null;
        }
        return sTextFromDataArea.toString();
    }
}
